# -*- python -*-
# ex: set syntax=python:

# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

####### LOCAL CONFIGURATION

from ConfigParser import RawConfigParser
import os
loc = RawConfigParser()
loc.read(os.path.join(os.path.dirname(__file__), 'local.ini'))

####### BUILDSLAVES

# The 'slaves' list defines the set of recognized buildslaves. Each element is
# a BuildSlave object, specifying a unique slave name and password.  The same
# slave name and password must be configured on the slave.
from buildbot.buildslave import BuildSlave
from buildbot.buildslave.ec2 import EC2LatentBuildSlave
c['slaves'] = []
slaves = {
    'el6-amd64': {
        'instance_type': 'm1.medium',
        'env': {
            'OPENSLIDE_VALGRIND_XFAIL': ','.join([
                'hamamatsu-ndpi-2012',
            ]),
        },
        'python3': None,
    },
    'el7-amd64': {
        # HVM instance
        'instance_type': 'm3.medium',
        'env': {
            'OPENSLIDE_TEST_XFAIL': ','.join([
                'aperio-33003',
                'aperio-jp2k-tile-zero-length',
            ]),
            'OPENSLIDE_VALGRIND_XFAIL': ','.join([
                'aperio-33003',
                'aperio-jp2k-tile-zero-length',
                'hamamatsu-ndpi-2012',
            ]),
        },
        'python3': None,
    },
    'f25-amd64': {
        'instance_type': 'm1.medium',
        'distcheck': True,
        'static_checks': True,
        'env': {
            'OPENSLIDE_VALGRIND_XFAIL': ','.join([
                'hamamatsu-ndpi-2012',
            ]),
        },
    },
    'f29-armv7l': {
        'parallel': '-j6',
        'valgrind': False,  # very slow; many unsuppressable false positives
    },
    'f25-ia32': {
        # Fedora 25 32-bit kernel won't boot on m1.medium
        'instance_type': 'm1.small',
        'env': {
            'OPENSLIDE_VALGRIND_XFAIL': ','.join([
                'hamamatsu-ndpi-2012',
            ]),
        },
    },
    'fbsd10-amd64': {
        'instance_type': 'm3.medium',
        'env': {
            'CPATH': '/usr/local/include',
            'LIBRARY_PATH': '/usr/local/lib',
        },
        'make': 'gmake',
        'python': 'python2',
        'coverage': False,  # not gcc
        'valgrind': False,  # unreliable
    },
    'jessie-ppc64': {
        'parallel': '-j3',
    },
    'macos10.12': {
        'env': {
            'CPATH': '/opt/local/include',
            'LIBRARY_PATH': '/opt/local/lib',
        },
        'python': 'python2.7',
        'python3': 'python3.5',
        'valgrind': False,  # no debug symbols; many problems in platform
        'coverage': False,  # not gcc
        'parallel': '-j3',
    },
    'winsrv2012r2-amd64': {
        'instance_type': 'c3.large',
        'env': {
            # Fix cairo builds with grep >= 2.23
            'LANG': 'en_US.UTF-8',
        },
        'parallel': '-j3',
        'windows': True,
        'winpython2': '/cygdrive/c/Python27/python',
        'winpython3': '/cygdrive/c/Program Files/Python35/python',
    },
}
for name, props in sorted(slaves.items()):
    password = loc.get('slaves', name)
    if 'instance_type' in props:
        slave = EC2LatentBuildSlave(name, password,
            properties=props,
            # Requires ami-filters.patch
            valid_ami_owners=[loc.getint('aws', 'account')],
            valid_ami_location_regex='[0-9]+/%s-[0-9]+$' % name,
            instance_type=props['instance_type'],
            identifier=loc.get('aws', 'access-key'),
            secret_identifier=loc.get('aws', 'secret-key'),
            region='us-east-1',
            keypair_name='openslide-buildslave',
            security_name='openslide-buildslave',
            # Requires instance-profile.patch
            instance_profile_name='openslide-buildslave',
            tags={
                'Name': 'latent-' + name,
            },
            notify_on_missing=loc.get('email', 'admins').split(','),
            spot_instance=True,
            # Requires max-spot-price.patch
            max_spot_price=0.10,
            price_multiplier=1000,
        )
    else:
        slave = BuildSlave(name, password, properties=props)
    c['slaves'].append(slave)

# 'protocols' contains information about protocols which master will use for
# communicating with slaves.
# You must define at least 'port' option that slaves could connect to your master
# with this protocol.
# 'port' must match the value configured into the buildslaves (with their
# --master option)
c['protocols'] = {'pb': {'port': 'tcp:9989:interface=127.0.0.1'}}

####### CHANGESOURCES

# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes.

c['change_source'] = []

####### BUILDERS

# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which slaves can execute them.  Note that any particular build will
# only take place on one slave.

from buildbot.locks import MasterLock, SlaveLock
from buildbot.process.factory import BuildFactory
from buildbot.process.logobserver import LogLineObserver
from buildbot.process.properties import Property, Interpolate, renderer
from buildbot.status.results import SUCCESS, SKIPPED
from buildbot.steps.master import MasterShellCommand, SetProperty
from buildbot.steps.source.git import Git
from buildbot.steps.shell import (ShellCommand, SetPropertyFromCommand,
        Configure, Compile, Test)
from buildbot.steps.slave import MakeDirectory, RemoveDirectory
from buildbot.steps.transfer import (FileDownload, FileUpload,
        MultipleFileUpload)
from buildbot.steps.trigger import Trigger

# Master locks

winbuild_upload_lock = MasterLock('winbuild-upload')

# hideStepIf callables

def hide_skipped(results, step):
    return results == SKIPPED

def hide_unless_error(results, step):
    return results in (SUCCESS, SKIPPED)

# Common property expansions

make = Property('make', default='make')
python = Property('python', default='python')
python3 = Property('python3', default='python3')
parallel = Property('parallel', default='-j2')

def build_result_url(item=''):
    return Interpolate('/results/%(prop:buildername)s/%(prop:buildnumber)s/' +
            item)

def build_result_path(item=''):
    return Interpolate('public_html' + build_result_url(item).fmtstring)

@renderer
def cache_if_ec2(props):
    if 'instance_type' in props:
        cache_script = (props['builddir'] +
                '/openslide-automation/scripts/caching-proxy.py')
        return [props.getProperty('python', 'python'), '-u', cache_script]
    else:
        return []

# Build steps

import re

class ShellCommandWithURLs(ShellCommand):
    parms = ShellCommand.parms + ['urls']
    renderables = ['urls']

    def finished(self, results):
        if results == SUCCESS:
            for name, url in sorted(self.urls.items()):
                if url:
                    self.addURL(name, url)
        ShellCommand.finished(self, results)

class OpenSlideTestSuite(Test):
    class TestSuiteLogObserver(LogLineObserver):
        numTests = 0
        numFailed = 0
        errorLog = None

        def outLineReceived(self, line):
            if self.errorLog and self.errorLog.isFinished():
                return
            # Count ANSI color resets as a rough progress metric
            if '\033[1;0m' in line:
                self.numTests += 1
                self.step.setProgress('tests', self.numTests)
            # Strip ANSI color codes
            line = re.sub(r'\033\[1;[0-9]+m', '', line)
            match = re.match('Failed: ([0-9]+)/[0-9]+$', line)
            if match:
                self.numFailed = int(match.group(1))
                if self.errorLog:
                    self.errorLog.finish()
            elif (not line.endswith(': OK') and
                    not line.endswith(': skipped') and
                    not line.endswith(': failed as expected')):
                if not self.errorLog:
                    self.errorLog = self.step.addLog('errors')
                if re.match('(==|--)[0-9]+(==|--)', line):
                    # Valgrind output
                    self.errorLog.addStdout(line + '\n')
                else:
                    self.errorLog.addStderr(line + '\n')

    def __init__(self, **kwargs):
        Test.__init__(self, **kwargs)
        self.observer = self.TestSuiteLogObserver()
        self.addLogObserver('stdio', self.observer)
        self.progressMetrics += ('tests',)

    def getText(self, cmd, results):
        ret = Test.getText(self, cmd, results)
        if self.observer.numFailed:
            ret += ['(%d errors)' % self.observer.numFailed]
        return ret

# Build factories

testsuite_factory = BuildFactory([
    Git(
        repourl='https://github.com/openslide/openslide',
        codebase='openslide',
        mode='full',
    ),
    Git(
        repourl='https://github.com/openslide/openslide-automation',
        codebase='openslide-automation',
        workdir='openslide-automation',
        mode='full',
    ),
    ShellCommand(
        command=['autoreconf', '-i'],
        name='autoreconf',
        description='autoconfiscating',
        descriptionDone='autoreconf',
        haltOnFailure=True,
    ),
    Configure(),
    Compile(
        command=[make, parallel, 'V=1'],
    ),
    ShellCommand(
        command=[make, 'distcheck'],
        name='distcheck',
        description='distchecking',
        descriptionDone='distcheck',
        doStepIf=lambda step: step.build.getProperty('distcheck', False),
        hideStepIf=hide_skipped,
    ),
    ShellCommand(
        command=[python, '-u', 'driver', 'exports'],
        workdir='openslide/test',
        name='exports',
        description='checking exports',
        descriptionDone='check exports',
        doStepIf=lambda step: step.build.getProperty('static_checks', False),
        hideStepIf=hide_skipped,
    ),
    ShellCommand(
        command=[cache_if_ec2, python, '-u', 'driver', 'unpack'],
        workdir='openslide/test',
        name='unpack',
        description='unpacking tests',
        descriptionDone='unpack',
        haltOnFailure=True,
    ),
    OpenSlideTestSuite(
        command=[python, '-u', 'driver', 'run'],
        workdir='openslide/test',
    ),
    OpenSlideTestSuite(
        command=[python, '-u', 'driver', 'valgrind'],
        workdir='openslide/test',
        name='valgrind',
        description='valgrinding',
        descriptionDone='valgrind',
        doStepIf=lambda step: step.build.getProperty('valgrind', True),
        hideStepIf=hide_skipped,
    ),
    ShellCommandWithURLs(
        command=[python, '-u', 'driver', 'coverage', 'coverage.txt'],
        workdir='openslide/test',
        name='coverage',
        description='checking coverage',
        descriptionDone='coverage',
        urls={
            'report': build_result_url('coverage.txt'),
        },
        doStepIf=lambda step: step.build.getProperty('coverage', True),
        hideStepIf=hide_skipped,
    ),
    FileUpload(
        slavesrc='test/coverage.txt',
        masterdest=build_result_path('coverage.txt'),
        mode=0644,
        doStepIf=lambda step: step.build.getProperty('coverage', True),
        hideStepIf=hide_unless_error,
        haltOnFailure=False,
    ),
    ShellCommandWithURLs(
        command=[python, '-u', 'driver', 'mosaic', 'mosaic.png'],
        workdir='openslide/test',
        name='mosaic',
        description='creating mosaic',
        descriptionDone='mosaic',
        urls={
            'result': build_result_url('mosaic.jpg'),
        },
    ),
    FileUpload(
        slavesrc='test/mosaic.png',
        masterdest=build_result_path('mosaic.png'),
        mode=0644,
        hideStepIf=hide_unless_error,
        haltOnFailure=False,
    ),
    MasterShellCommand(
        command=Interpolate('''
            cd '%s' &&
            convert -quality 90 mosaic.png mosaic.jpg &&
            rm mosaic.png
        ''' % build_result_path().fmtstring),
        name='mosaic-compress',
        description='compressing mosaic',
        descriptionDone='compress mosaic',
        hideStepIf=hide_unless_error,
    ),
    ShellCommand(
        command=[python, '-u', 'driver', 'time'],
        workdir='openslide/test',
        name='time',
        description='timing',
        descriptionDone='time',
    ),
])
testsuite_factory.workdir = 'openslide'

wintest_factory = BuildFactory([
    # Checkout
    Git(
        repourl='https://github.com/openslide/openslide-winbuild',
        codebase='openslide-winbuild',
        workdir='openslide-winbuild',
        mode='full',
    ),
    MakeDirectory(
        dir='openslide-winbuild/override',
        description='creating override directory',
        hideStepIf=hide_unless_error,
    ),
    Git(
        repourl='https://github.com/openslide/openslide',
        codebase='openslide',
        workdir='openslide-winbuild/override/openslide',
        mode='full',
    ),
    Git(
        repourl='https://github.com/openslide/openslide-automation',
        codebase='openslide-automation',
        workdir='openslide-automation',
        mode='full',
    ),

    # Build
    ShellCommand(
        command=[cache_if_ec2, './build.sh', 'sdist'],
        workdir='openslide-winbuild',
        name='fetch-deps',
        description='fetching dependencies',
        descriptionDone='fetch dependencies',
        haltOnFailure=True,
    ),
    ShellCommand(
        command=['autoreconf', '-i'],
        workdir='openslide-winbuild/override/openslide',
        name='autoreconf',
        description='autoconfiscating',
        descriptionDone='autoreconf',
        haltOnFailure=True,
    ),
    ShellCommand(
        command=['./build.sh', parallel, Interpolate('-m%(prop:bitness)s'),
                'bdist'],
        workdir='openslide-winbuild',
        name='build',
        description='building',
        descriptionDone='build',
        haltOnFailure=True,
    ),

    # Test
    SetPropertyFromCommand(
        command=['sh', '-c', 'echo openslide-[0-9]*'],
        property='openslide_builddir',
        workdir=Interpolate('openslide-winbuild/%(prop:bitness)s/build'),
        name='get-builddir',
        description='getting builddir',
        descriptionDone='get builddir',
        hideStepIf=hide_unless_error,
        haltOnFailure=True,
    ),
    ShellCommand(
        command=[cache_if_ec2, python, '-u', 'driver', 'unpack'],
        name='unpack',
        description='unpacking tests',
        descriptionDone='unpack',
        haltOnFailure=True,
    ),
    OpenSlideTestSuite(
        command=[python, '-u', 'driver', 'run'],
    ),
    ShellCommandWithURLs(
        command=[python, '-u', 'driver', 'mosaic', 'mosaic.png'],
        name='mosaic',
        description='creating mosaic',
        descriptionDone='mosaic',
        urls={
            'result': build_result_url('mosaic.jpg'),
        },
    ),
    FileUpload(
        slavesrc='mosaic.png',
        masterdest=build_result_path('mosaic.png'),
        mode=0644,
        hideStepIf=hide_unless_error,
        haltOnFailure=False,
    ),
    MasterShellCommand(
        command=Interpolate('''
            cd '%s' &&
            convert -quality 90 mosaic.png mosaic.jpg &&
            rm mosaic.png
        ''' % build_result_path().fmtstring),
        name='mosaic-compress',
        description='compressing mosaic',
        descriptionDone='compress mosaic',
        hideStepIf=hide_unless_error,
    ),
    ShellCommand(
        command=[python, '-u', 'driver', 'time'],
        name='time',
        description='timing',
        descriptionDone='time',
    ),

    # Clean up test data
    RemoveDirectory(
        dir=Interpolate('openslide-winbuild/%(prop:bitness)s/' +
                'build/%(prop:openslide_builddir)s/test/_slidedata'),
        name='cleanup',
        description='cleaning up data',
        descriptionDone='clean up data',
        hideStepIf=hide_unless_error,
        alwaysRun=True,
    ),
])
wintest_factory.workdir = Interpolate('openslide-winbuild/%(prop:bitness)s/' +
        'build/%(prop:openslide_builddir)s/test')

pytest_env = {
    'DYLD_LIBRARY_PATH': Interpolate('%(prop:workdir)s/install/lib'),
    'LD_LIBRARY_PATH': Interpolate('%(prop:workdir)s/install/lib'),
}
pytest_factory = BuildFactory([
    Git(
        repourl='https://github.com/openslide/openslide',
        codebase='openslide',
        workdir='openslide',
        mode='full',
    ),
    Git(
        repourl='https://github.com/openslide/openslide-python',
        codebase='openslide-python',
        mode='full',
    ),
    RemoveDirectory(
        dir='install',
        name='clean',
        description='cleaning install dir',
        descriptionDone='clean install dir',
        hideStepIf=hide_unless_error,
    ),
    ShellCommand(
        command=['autoreconf', '-i'],
        name='autoreconf-os',
        workdir='openslide',
        description='autoconfiscating openslide',
        descriptionDone='autoreconf openslide',
        haltOnFailure=True,
    ),
    Configure(
        command=['./configure',
                Interpolate('--prefix=%(prop:workdir)s/install')],
        name='configure-os',
        workdir='openslide',
        description='configuring openslide',
        descriptionDone='configure openslide',
    ),
    Compile(
        command=[make, parallel, 'V=1'],
        name='build-os',
        workdir='openslide',
        description='building openslide',
        descriptionDone='build openslide',
    ),
    ShellCommand(
        command=[make, 'install'],
        name='install-os',
        workdir='openslide',
        description='installing openslide',
        descriptionDone='install openslide',
        haltOnFailure=True,
    ),
    # make "setup.py test" use installed PIL rather than installing Pillow
    ShellCommand(
        command=['sed', '-i.old', '/Pillow/d', 'setup.py'],
        name='filter-pillow',
        description='filtering Pillow dep',
        descriptionDone='filter Pillow dep',
        hideStepIf=hide_unless_error,
        haltOnFailure=True,
    ),
    Test(
        command=[python, '-u', 'setup.py', '--without-performance', 'test'],
        name='test-py2-slow',
        env=pytest_env,
        description='testing py2 slow path',
        descriptionDone='test py2 slow path',
    ),
    Test(
        command=[python, '-u', 'setup.py', 'test'],
        name='test-py2',
        env=pytest_env,
        description='testing py2',
        descriptionDone='test py2',
    ),
    Test(
        command=[python3, '-u', 'setup.py', '--without-performance', 'test'],
        name='test-py3-slow',
        env=pytest_env,
        description='testing py3 slow path',
        descriptionDone='test py3 slow path',
        doStepIf=lambda step: bool(step.build.getProperty('python3', True)),
        hideStepIf=hide_skipped,
    ),
    Test(
        command=[python3, '-u', 'setup.py', 'test'],
        name='test-py3',
        env=pytest_env,
        description='testing py3',
        descriptionDone='test py3',
        doStepIf=lambda step: bool(step.build.getProperty('python3', True)),
        hideStepIf=hide_skipped,
    ),
])
pytest_factory.workdir = 'openslide-python'

def get_latest_windows_snapshot_name(artifact):
    @renderer
    def func(props):
        prog = re.compile('openslide-%s-.*\.zip$' % artifact)
        return sorted([n for n in os.listdir('public_html/snapshots/windows')
                if prog.match(n)])[-1].rstrip('.zip')
    return func
winpytest_env = {
    'PATH': [
        Interpolate('%(prop:workdir)s/snapshot/%(prop:snapshot)s/bin'),
        '${PATH}',
    ],
}
winpytest_factory = BuildFactory([
    SetProperty(
        property='snapshot',
        value=get_latest_windows_snapshot_name('win64'),
        hideStepIf=hide_unless_error,
        locks=[
            winbuild_upload_lock.access('counting'),
        ],
    ),
    Git(
        repourl='https://github.com/openslide/openslide-python',
        codebase='openslide-python',
        mode='full',
    ),
    RemoveDirectory(
        dir='snapshot',
        name='clean',
        description='cleaning snapshot dir',
        descriptionDone='clean snapshot dir',
        hideStepIf=hide_unless_error,
    ),
    FileDownload(
        mastersrc=Interpolate('public_html/snapshots/windows/' +
                '%(prop:snapshot)s.zip'),
        slavedest=Interpolate('snapshot/%(prop:snapshot)s.zip'),
        workdir='',
        hideStepIf=hide_unless_error,
    ),
    ShellCommand(
        command=['unzip', Interpolate('%(prop:snapshot)s.zip')],
        name='unpack',
        workdir='snapshot',
        description='unpacking binaries',
        descriptionDone='unpack binaries',
        haltOnFailure=True,
        hideStepIf=hide_unless_error,
    ),
    Test(
        command=[Property('winpython2'), '-u', 'setup.py',
                '--without-performance', 'test'],
        name='test-py2-slow',
        env=winpytest_env,
        description='testing py2 slow path',
        descriptionDone='test py2 slow path',
    ),
    Test(
        command=[Property('winpython3'), '-u', 'setup.py',
                '--without-performance', 'test'],
        name='test-py3-slow',
        env=winpytest_env,
        description='testing py3 slow path',
        descriptionDone='test py3 slow path',
    ),
])
winpytest_factory.workdir = 'openslide-python'

from datetime import date
import pipes
import sys
@renderer
def winbuild_pkgver(props):
    pkgver = date.today().strftime('%Y%m%d')
    if props['nightly']:
        pkgver += '-nightly'
    return pkgver
@renderer
def winbuild_args(props):
    args = ['-p' + props['pkgver']]
    if props['nightly']:
        args.append('-sg' + props['got_revision']['openslide'][:7])
    return args
def winbuild_make_url(artifact):
    @renderer
    def func(props):
        filename = 'openslide-%s-%s.zip' % (artifact, props['pkgver'])
        if props['nightly']:
            return '/snapshots/windows/%s' % filename
        else:
            return '/results/%s/%s/%s' % (props['buildername'],
                    props['buildnumber'], filename)
    return func
@renderer
def winbuild_destpath(props):
    if props['nightly']:
        return 'public_html/snapshots/windows/'
    else:
        return 'public_html/results/%s/%s/' % (props['buildername'],
                props['buildnumber'])
@renderer
def winbuild_index_command(props):
    if props['nightly']:
        return ' '.join([
            sys.executable,
            '../scripts/winbuild-index.py',
            'public_html/snapshots/windows',
            props['pkgver'],
            pipes.quote(props['got_revision']['openslide']),
            pipes.quote(props['got_revision']['openslide-java']),
            pipes.quote(props['got_revision']['openslide-winbuild']),
        ])
    else:
        # The returned command should never run, but we still need to
        # special-case this because some of the revision lookups would
        # otherwise raise KeyError
        return 'false'
winbuild_osj_env = {
    'PKG_CONFIG_PATH': Interpolate('%(prop:workdir)s/install/lib/pkgconfig'),
}
winbuild_test_slide = 'http://openslide.cs.cmu.edu/download/openslide-testdata/Aperio/CMU-1-Small-Region.svs'
winbuild_factory = BuildFactory([
    # Prep
    SetProperty(
        property='pkgver',
        value=winbuild_pkgver,
        hideStepIf=hide_unless_error,
    ),
    Git(
        repourl='https://github.com/openslide/openslide',
        codebase='openslide',
        workdir='openslide',
        mode='full',
        doStepIf=lambda step: step.build.getProperty('nightly'),
        hideStepIf=hide_skipped,
    ),
    Git(
        repourl='https://github.com/openslide/openslide-java',
        codebase='openslide-java',
        workdir='openslide-java',
        mode='full',
        doStepIf=lambda step: step.build.getProperty('nightly'),
        hideStepIf=hide_skipped,
    ),
    Git(
        repourl='https://github.com/openslide/openslide-winbuild',
        codebase='openslide-winbuild',
        mode='full',
    ),
    Git(
        repourl='https://github.com/openslide/openslide-automation',
        codebase='openslide-automation',
        workdir='openslide-automation',
        mode='full',
    ),
    RemoveDirectory(
        dir='install',
        name='clean',
        description='cleaning install dir',
        descriptionDone='clean install dir',
        hideStepIf=hide_unless_error,
    ),

    # Generate OpenSlide tarball if nightly build
    ShellCommand(
        command=['autoreconf', '-i'],
        name='autoreconf-os',
        workdir='openslide',
        description='autoconfiscating openslide',
        descriptionDone='autoreconf openslide',
        haltOnFailure=True,
        doStepIf=lambda step: step.build.getProperty('nightly'),
        hideStepIf=hide_skipped,
    ),
    Configure(
        command=['./configure',
                Interpolate('--prefix=%(prop:workdir)s/install')],
        name='configure-os',
        workdir='openslide',
        description='configuring openslide',
        descriptionDone='configure openslide',
        doStepIf=lambda step: step.build.getProperty('nightly'),
        hideStepIf=hide_skipped,
    ),
    ShellCommand(
        command=[make, 'dist'],
        name='dist-os',
        workdir='openslide',
        description='disting openslide',
        descriptionDone='dist openslide',
        haltOnFailure=True,
        doStepIf=lambda step: step.build.getProperty('nightly'),
        hideStepIf=hide_skipped,
    ),
    # We need to install so OpenSlide Java can find us
    Compile(
        command=[make, parallel, 'V=1'],
        name='build-os',
        workdir='openslide',
        description='building openslide',
        descriptionDone='build openslide',
        doStepIf=lambda step: step.build.getProperty('nightly'),
        hideStepIf=hide_skipped,
    ),
    ShellCommand(
        command=[make, 'install'],
        name='install-os',
        workdir='openslide',
        description='installing openslide',
        descriptionDone='install openslide',
        haltOnFailure=True,
        doStepIf=lambda step: step.build.getProperty('nightly'),
        hideStepIf=hide_skipped,
    ),

    # Generate OpenSlide Java tarball if nightly build
    ShellCommand(
        command=['autoreconf', '-i'],
        name='autoreconf-osj',
        workdir='openslide-java',
        description='autoconfiscating openslide-java',
        descriptionDone='autoreconf openslide-java',
        haltOnFailure=True,
        doStepIf=lambda step: step.build.getProperty('nightly'),
        hideStepIf=hide_skipped,
    ),
    Configure(
        command=['./configure'],
        name='configure-osj',
        workdir='openslide-java',
        env=winbuild_osj_env,
        description='configuring openslide-java',
        descriptionDone='configure openslide-java',
        doStepIf=lambda step: step.build.getProperty('nightly'),
        hideStepIf=hide_skipped,
    ),
    ShellCommand(
        command=[make, 'dist'],
        name='dist-osj',
        workdir='openslide-java',
        env=winbuild_osj_env,
        description='disting openslide-java',
        descriptionDone='dist openslide-java',
        haltOnFailure=True,
        doStepIf=lambda step: step.build.getProperty('nightly'),
        hideStepIf=hide_skipped,
    ),

    # Build
    MakeDirectory(
        dir='openslide-winbuild/tar',
        description='creating tar directory',
        hideStepIf=hide_unless_error,
    ),
    ShellCommand(
        # Use shell for wildcards
        command=['sh', '-c', 'cp ../openslide/openslide-*.tar.xz ' +
                '../openslide-java/openslide-java-*.tar.xz tar'],
        name='copy-tar',
        description='copying tarballs',
        descriptionDone='copy tarballs',
        haltOnFailure=True,
        doStepIf=lambda step: step.build.getProperty('nightly'),
        hideStepIf=hide_unless_error,
    ),
    ShellCommandWithURLs(
        command=[cache_if_ec2, './build.sh', winbuild_args, 'sdist'],
        name='sdist',
        description='sdist',
        haltOnFailure=True,
        urls={
            'zip': winbuild_make_url('winbuild'),
        },
    ),
    ShellCommandWithURLs(
        command=['./build.sh', winbuild_args, parallel, '-m32', 'bdist'],
        name='bdist32',
        description='32-bit bdist',
        haltOnFailure=True,
        urls={
            'zip': winbuild_make_url('win32'),
        },
    ),
    ShellCommandWithURLs(
        command=['./build.sh', winbuild_args, parallel, '-m64', 'bdist'],
        name='bdist64',
        description='64-bit bdist',
        haltOnFailure=True,
        urls={
            'zip': winbuild_make_url('win64'),
        },
    ),

    # Smoke test
    MakeDirectory(
        dir='test',
        description='creating test directory',
        hideStepIf=hide_unless_error,
    ),
    ShellCommand(
        command=[cache_if_ec2, 'wget', '-q', winbuild_test_slide],
        name='fetch-slide',
        workdir='test',
        description='fetching slide',
        descriptionDone='fetch slide',
        haltOnFailure=True,
        hideStepIf=hide_unless_error,
    ),
    ShellCommand(
        command=['unzip', Interpolate('../openslide-winbuild/openslide-win32-%(prop:pkgver)s.zip')],
        name='unpack-32',
        workdir='test',
        description='unpacking 32-bit binaries',
        descriptionDone='unpack 32-bit binaries',
        haltOnFailure=True,
        hideStepIf=hide_unless_error,
    ),
    ShellCommand(
        command=['unzip', Interpolate('../openslide-winbuild/openslide-win64-%(prop:pkgver)s.zip')],
        name='unpack-64',
        workdir='test',
        description='unpacking 64-bit binaries',
        descriptionDone='unpack 64-bit binaries',
        haltOnFailure=True,
        hideStepIf=hide_unless_error,
    ),
    ShellCommand(
        command=['wine', 'openslide-write-png.exe',
                '../../' + winbuild_test_slide.split('/')[-1],
                '0', '0', '0', '100', '100', 'out.png'],
        name='check-32',
        workdir=Interpolate('test/openslide-win32-%(prop:pkgver)s/bin'),
        description='checking 32-bit',
        descriptionDone='check 32-bit',
        haltOnFailure=True,
    ),
    ShellCommand(
        command=['wine', 'openslide-write-png.exe',
                '../../' + winbuild_test_slide.split('/')[-1],
                '0', '0', '0', '100', '100', 'out.png'],
        name='check-64',
        workdir=Interpolate('test/openslide-win64-%(prop:pkgver)s/bin'),
        description='checking 64-bit',
        descriptionDone='check 64-bit',
        haltOnFailure=True,
    ),

    # Upload
    MultipleFileUpload(
        slavesrcs=[
            Interpolate('openslide-winbuild/openslide-winbuild-%(prop:pkgver)s.zip'),
            Interpolate('openslide-winbuild/openslide-win32-%(prop:pkgver)s.zip'),
            Interpolate('openslide-winbuild/openslide-win64-%(prop:pkgver)s.zip'),
        ],
        masterdest=winbuild_destpath,
        # http://trac.buildbot.net/ticket/3009
        workdir='',
        mode=0644,
        hideStepIf=hide_unless_error,
        locks=[
            winbuild_upload_lock.access('exclusive'),
        ],
    ),
    MasterShellCommand(
        command=winbuild_index_command,
        name='index',
        description='indexing',
        descriptionDone='index',
        doStepIf=lambda step: step.build.getProperty('nightly'),
        hideStepIf=hide_unless_error,
    ),

    # Trigger other builders
    Trigger(
        schedulerNames=[
            'winpytest-nightly-winbuild',
        ],
        alwaysUseLatest=True,
        doStepIf=lambda step: step.build.getProperty('nightly'),
        hideStepIf=hide_unless_error,
    ),
])
winbuild_factory.workdir = 'openslide-winbuild'

retile_osp_env = {
    'LD_LIBRARY_PATH': Interpolate('%(prop:workdir)s/install/lib'),
    'PYTHONPATH': Interpolate('%(prop:workdir)s/install/lib/python'),
}
retile_factory = BuildFactory([
    # Prep
    Git(
        repourl='https://github.com/openslide/openslide',
        codebase='openslide',
        mode='full',
    ),
    Git(
        repourl='https://github.com/openslide/openslide-python',
        codebase='openslide-python',
        workdir='openslide-python',
        mode='full',
    ),
    Git(
        repourl='https://github.com/openslide/openslide.github.com',
        codebase='openslide.github.com',
        workdir='openslide.github.com',
        mode='full',
    ),
    Git(
        repourl='https://github.com/openslide/openslide-automation',
        codebase='openslide-automation',
        workdir='openslide-automation',
        mode='full',
    ),
    RemoveDirectory(
        dir='install',
        name='clean',
        description='cleaning install dir',
        descriptionDone='clean install dir',
        hideStepIf=hide_unless_error,
    ),

    # Build/install OpenSlide
    ShellCommand(
        command=['autoreconf', '-i'],
        name='autoreconf-os',
        description='autoconfiscating openslide',
        descriptionDone='autoreconf openslide',
        haltOnFailure=True,
    ),
    Configure(
        command=['./configure',
                Interpolate('--prefix=%(prop:workdir)s/install')],
        name='configure-os',
        description='configuring openslide',
        descriptionDone='configure openslide',
    ),
    Compile(
        command=[make, parallel, 'V=1'],
        name='build-os',
        description='building openslide',
        descriptionDone='build openslide',
    ),
    ShellCommand(
        command=[make, 'install'],
        name='install-os',
        description='installing openslide',
        descriptionDone='install openslide',
        haltOnFailure=True,
    ),

    # Install OpenSlide Python
    MakeDirectory(
        dir='install/lib/python',
        description='creating Python directory',
        descriptionDone='create Python directory',
        hideStepIf=hide_unless_error,
    ),
    ShellCommand(
        command=[python, '-u', 'setup.py', 'install',
                Interpolate('--home=%(prop:workdir)s/install'),
                Interpolate('--install-platlib=%(prop:workdir)s/install/lib/python')],
        name='install-osp',
        workdir='openslide-python',
        env=retile_osp_env,
        description='installing openslide-python',
        descriptionDone='install openslide-python',
        haltOnFailure=True,
    ),

    # Tile
    ShellCommand(
        command=[cache_if_ec2, python, '-u', '_synctiles.py', parallel],
        name='tile',
        workdir='openslide.github.com/demo',
        env=retile_osp_env,
        description='tiling',
        descriptionDone='tile',
        haltOnFailure=True,
    ),
])
retile_factory.workdir = 'openslide'

from buildbot.config import BuilderConfig

# Prevent 32-bit and 64-bit builds from having their testcases unpacked
# simultaneously
wintest_slave_lock = SlaveLock('wintest')

singleton_slavename = 'f25-amd64'

c['builders'] = [
    BuilderConfig(
        name='retile',
        tags=['website'],
        slavename=singleton_slavename,
        factory=retile_factory,
        env=slaves[singleton_slavename].get('env'),
    ),
]

# winbuild-* builders
for _suffix, _nightly in ('nightly', True), ('release', False):
    c['builders'].append(BuilderConfig(
        name='winbuild-%s' % _suffix,
        tags=['winbuild'],
        slavename=singleton_slavename,
        factory=winbuild_factory,
        env=slaves[singleton_slavename].get('env'),
        properties={
            'nightly': _nightly,
        },
    ))

for slave in c['slaves']:
    if slaves[slave.slavename].get('windows', False):
        # wintest-* builders
        if slaves[slave.slavename].get('testsuite', True):
            c['builders'].append(BuilderConfig(
                name='wintest-' + slave.slavename + '-32',
                tags=['wintest'],
                slavename=slave.slavename,
                factory=wintest_factory,
                env=slaves[slave.slavename].get('env'),
                properties={
                    'bitness': '32',
                },
                locks=[
                    wintest_slave_lock.access('counting'),
                ],
            ))
            c['builders'].append(BuilderConfig(
                name='wintest-' + slave.slavename + '-64',
                tags=['wintest'],
                slavename=slave.slavename,
                factory=wintest_factory,
                env=slaves[slave.slavename].get('env'),
                properties={
                    'bitness': '64',
                },
                locks=[
                    wintest_slave_lock.access('counting'),
                ],
            ))
        # winpytest-* builders
        c['builders'].append(BuilderConfig(
            name='winpytest-' + slave.slavename,
            tags=['winpytest'],
            slavename=slave.slavename,
            factory=winpytest_factory,
            env=slaves[slave.slavename].get('env'),
        ))
    else:
        # testsuite-* builders
        if slaves[slave.slavename].get('testsuite', True):
            c['builders'].append(BuilderConfig(
                name='testsuite-' + slave.slavename,
                tags=['testsuite'],
                slavename=slave.slavename,
                factory=testsuite_factory,
                env=slaves[slave.slavename].get('env'),
            ))
        # pytest-* builders
        c['builders'].append(BuilderConfig(
            name='pytest-' + slave.slavename,
            tags=['pytest'],
            slavename=slave.slavename,
            factory=pytest_factory,
            env=slaves[slave.slavename].get('env'),
        ))

# Use adjacent priorities for builders on the same buildslave.  This
# prevents multiple pending builders on a latent buildslave from causing a
# VM launch, build, VM termination, and another launch later on.
def prioritizeBuilders(master, builders):
    return sorted(builders,
            key=lambda b: (sorted(b.config.slavenames)[0], b.name))
c['prioritizeBuilders'] = prioritizeBuilders

####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes.

from buildbot.schedulers.forcesched import (ForceScheduler, FixedParameter,
        StringParameter, CodebaseParameter)
from buildbot.schedulers.timed import Nightly
from buildbot.schedulers.triggerable import Triggerable
from buildbot.changes import filter

c['schedulers'] = [
    Nightly(
        name='testsuite-nightly',
        builderNames=[
            b.name for b in c['builders'] if b.name.startswith('testsuite-')
        ],
        change_filter=filter.ChangeFilter(
            codebase='openslide',
            branch='master',
        ),
        codebases={
            'openslide': {
                'repository': 'https://github.com/openslide/openslide',
                'branch': 'master',
            },
            'openslide-automation': {
                'repository': 'https://github.com/openslide/openslide-automation',
                'branch': 'master',
            },
        },
        branch='master',  # required
        hour=5,
        onlyIfChanged=True,
    ),
    ForceScheduler(
        name='testsuite-manual',
        builderNames=[
            b.name for b in c['builders'] if b.name.startswith('testsuite-')
        ],
        codebases=[
            CodebaseParameter(
                'openslide',
                repository=FixedParameter('repository',
                        default='https://github.com/openslide/openslide'),
                branch=FixedParameter('branch', default='master'),
                project=None,
                revision=None,
            ),
            CodebaseParameter(
                'openslide-automation',
                repository=FixedParameter('repository',
                        default='https://github.com/openslide/openslide-automation'),
                branch=FixedParameter('branch', default='master'),
                project=None,
                revision=None,
            ),
        ],
        properties=[],
        buttonName='Start',
    ),

    Nightly(
        name='wintest-nightly',
        builderNames=[
            b.name for b in c['builders'] if b.name.startswith('wintest-')
        ],
        change_filter=filter.ChangeFilter(
            codebase=['openslide', 'openslide-winbuild'],
            branch='master',
        ),
        codebases={
            'openslide': {
                'repository': 'https://github.com/openslide/openslide',
                'branch': 'master',
            },
            'openslide-winbuild': {
                'repository': 'https://github.com/openslide/openslide-winbuild',
                'branch': 'master',
            },
            'openslide-automation': {
                'repository': 'https://github.com/openslide/openslide-automation',
                'branch': 'master',
            },
        },
        branch='master',  # required
        hour=5,
        onlyIfChanged=True,
    ),
    ForceScheduler(
        name='wintest-manual',
        builderNames=[
            b.name for b in c['builders'] if b.name.startswith('wintest-')
        ],
        codebases=[
            CodebaseParameter(
                'openslide',
                repository=FixedParameter('repository',
                        default='https://github.com/openslide/openslide'),
                branch=StringParameter(
                    name='branch',
                    label='Branch/Tag/Commit:',
                    default='master',
                ),
                project=None,
                revision=None,
            ),
            CodebaseParameter(
                'openslide-winbuild',
                repository=FixedParameter('repository',
                        default='https://github.com/openslide/openslide-winbuild'),
                branch=FixedParameter('branch', default='master'),
                project=None,
                revision=None,
            ),
            CodebaseParameter(
                'openslide-automation',
                repository=FixedParameter('repository',
                        default='https://github.com/openslide/openslide-automation'),
                branch=FixedParameter('branch', default='master'),
                project=None,
                revision=None,
            ),
        ],
        properties=[],
        buttonName='Start',
    ),

    Nightly(
        name='pytest-nightly',
        builderNames=[
            b.name for b in c['builders'] if b.name.startswith('pytest-')
        ],
        change_filter=filter.ChangeFilter(
            codebase=['openslide', 'openslide-python'],
            branch='master',
        ),
        codebases={
            'openslide': {
                'repository': 'https://github.com/openslide/openslide',
                'branch': 'master',
            },
            'openslide-python': {
                'repository': 'https://github.com/openslide/openslide-python',
                'branch': 'master',
            },
        },
        branch='master',  # required
        hour=5,
        onlyIfChanged=True,
    ),
    ForceScheduler(
        name='pytest-manual',
        builderNames=[
            b.name for b in c['builders'] if b.name.startswith('pytest-')
        ],
        codebases=[
            CodebaseParameter(
                'openslide',
                repository=FixedParameter('repository',
                        default='https://github.com/openslide/openslide'),
                branch=FixedParameter('branch', default='master'),
                project=None,
                revision=None,
            ),
            CodebaseParameter(
                'openslide-python',
                repository=FixedParameter('repository',
                        default='https://github.com/openslide/openslide-python'),
                branch=FixedParameter('branch', default='master'),
                project=None,
                revision=None,
            ),
        ],
        properties=[],
        buttonName='Start',
    ),

    Nightly(
        # Rebuild whenever openslide-python is updated
        name='winpytest-nightly-python',
        builderNames=[
            b.name for b in c['builders'] if b.name.startswith('winpytest-')
        ],
        change_filter=filter.ChangeFilter(
            codebase='openslide-python',
            branch='master',
        ),
        codebases={
            'openslide-python': {
                'repository': 'https://github.com/openslide/openslide-python',
                'branch': 'master',
            },
        },
        branch='master',  # required
        hour=5,
        onlyIfChanged=True,
    ),
    Triggerable(
        # Also rebuild when there's a new Windows nightly build
        name='winpytest-nightly-winbuild',
        builderNames=[
            b.name for b in c['builders'] if b.name.startswith('winpytest-')
        ],
        codebases={
            'openslide-python': {
                'repository': 'https://github.com/openslide/openslide-python',
                'branch': 'master',
            },
        },
    ),
    ForceScheduler(
        name='winpytest-manual',
        builderNames=[
            b.name for b in c['builders'] if b.name.startswith('winpytest-')
        ],
        codebases=[
            CodebaseParameter(
                'openslide-python',
                repository=FixedParameter('repository',
                        default='https://github.com/openslide/openslide-python'),
                branch=FixedParameter('branch', default='master'),
                project=None,
                revision=None,
            ),
        ],
        properties=[],
        buttonName='Start',
    ),

    Nightly(
        name='winbuild-nightly',
        builderNames=['winbuild-nightly'],
        change_filter=filter.ChangeFilter(
            codebase=['openslide', 'openslide-java', 'openslide-winbuild'],
            branch='master',
        ),
        codebases={
            'openslide': {
                'repository': 'https://github.com/openslide/openslide',
                'branch': 'master',
            },
            'openslide-java': {
                'repository': 'https://github.com/openslide/openslide-java',
                'branch': 'master',
            },
            'openslide-winbuild': {
                'repository': 'https://github.com/openslide/openslide-winbuild',
                'branch': 'master',
            },
            'openslide-automation': {
                'repository': 'https://github.com/openslide/openslide-automation',
                'branch': 'master',
            },
        },
        branch='master',  # required
        hour=5,
        onlyIfChanged=True,
    ),
    # No forcing of nightly winbuilds: would lead to datestamp collisions

    ForceScheduler(
        name='winbuild-release',
        builderNames=['winbuild-release'],
        codebases=[
            CodebaseParameter(
                'openslide-winbuild',
                repository=FixedParameter('repository',
                        default='https://github.com/openslide/openslide-winbuild'),
                branch=FixedParameter('branch', default='master'),
                project=None,
                revision=None,
            ),
            CodebaseParameter(
                'openslide-automation',
                repository=FixedParameter('repository',
                        default='https://github.com/openslide/openslide-automation'),
                branch=FixedParameter('branch', default='master'),
                project=None,
                revision=None,
            ),
        ],
        properties=[],
        buttonName='Start',
    ),

    ForceScheduler(
        name='retile',
        builderNames=['retile'],
        codebases=[
            CodebaseParameter(
                'openslide',
                repository=FixedParameter('repository',
                        default='https://github.com/openslide/openslide'),
                branch=StringParameter(
                    name='branch',
                    label='Tag:',
                    regex=r'^v[0-9.]+$',
                ),
                project=None,
                revision=None,
            ),
            CodebaseParameter(
                'openslide-python',
                repository=FixedParameter('repository',
                        default='https://github.com/openslide/openslide-python'),
                branch=StringParameter(
                    name='branch',
                    label='Tag:',
                    regex=r'^v[0-9.]+$',
                ),
                project=None,
                revision=None,
            ),
            CodebaseParameter(
                'openslide.github.com',
                repository=FixedParameter('repository',
                        default='https://github.com/openslide/openslide.github.com'),
                branch=FixedParameter('branch', default='master'),
                project=None,
                revision=None,
            ),
            CodebaseParameter(
                'openslide-automation',
                repository=FixedParameter('repository',
                        default='https://github.com/openslide/openslide-automation'),
                branch=FixedParameter('branch', default='master'),
                project=None,
                revision=None,
            ),
        ],
        properties=[],
        buttonName='Start',
    ),
]

####### STATUS TARGETS

# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.

from buildbot.status import html
from buildbot.status.mail import MailNotifier
from buildbot.status.web.auth import HTPasswdAprAuth
from buildbot.status.web.authz import Authz

class CustomAuth(HTPasswdAprAuth):
    def getUserInfo(self, user):
        info = HTPasswdAprAuth.getUserInfo(self, user)
        # Override default "<user>@localhost"
        info['email'] = user
        return info

c['status'] = [
    html.WebStatus(
        http_port='tcp:8010:interface=127.0.0.1',
        authz=Authz(
            auth=CustomAuth(os.path.join(os.path.dirname(__file__), 'users')),
            forceBuild='auth',
            forceAllBuilds='auth',
            stopBuild='auth',
            stopAllBuilds='auth',
            cancelPendingBuild='auth',
        ),
        change_hook_dialects={
            'github': {
                'secret': loc.get('github', 'secret'),
                'strict': True,
            },
        },
        # Username/password transmitted via HTTP Basic.  Extra layer of
        # authentication, in addition to the webhook secret.  Password should
        # be different from the secret above.
        change_hook_auth=['file:' + os.path.join(os.path.dirname(__file__),
                'changehook.password')],
    ),
    MailNotifier(
        tags=['testsuite', 'wintest', 'pytest', 'winpytest'],
        fromaddr='openslide@openslide.org',
        sendToInterestedUsers=False,
        extraRecipients=loc.get('email', 'test-runs').split(','),
        mode='warnings',
    ),
    MailNotifier(
        builders=['winbuild-nightly'],
        fromaddr='openslide@openslide.org',
        sendToInterestedUsers=False,
        extraRecipients=loc.get('email', 'snapshot-builds').split(','),
        mode='warnings',
    ),
    MailNotifier(
        builders=['retile', 'winbuild-release'],
        fromaddr='openslide@openslide.org',
        sendToInterestedUsers=False,
        extraRecipients=loc.get('email', 'release-tasks').split(','),
        mode='all',
    ),
]

####### PROJECT IDENTITY

# the 'title' string will appear at the top of this buildbot
# installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.

c['title'] = "OpenSlide"
c['titleURL'] = "https://openslide.org/"

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = "https://buildbot.openslide.org/"

####### DB URL

c['db'] = {
    # This specifies what database buildbot uses to store its state.  You can leave
    # this at its default for all but the largest installations.
    'db_url' : "sqlite:///state.sqlite",
}

####### CODEBASES

all_repositories = {
    'https://github.com/openslide/openslide': 'openslide',
    'https://github.com/openslide/openslide-java': 'openslide-java',
    'https://github.com/openslide/openslide-python': 'openslide-python',
    'https://github.com/openslide/openslide-winbuild': 'openslide-winbuild',
    'https://github.com/openslide/openslide-automation': 'openslide-automation',
    'https://github.com/openslide/openslide.github.com': 'openslide.github.com',
}

def codebaseGenerator(chdict):
    return all_repositories[chdict['repository']]
c['codebaseGenerator'] = codebaseGenerator

####### OTHER CONFIGURATION

c['caches'] = {
    'Changes' : 1000,
}
